"""
Copyright (c) 2022 Huawei Technologies Co.,Ltd.

openGauss is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:

          http://license.coscl.org.cn/MulanPSL2

THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details.
"""
"""
Case Type   : 极致RTO-数据库系统类
Case Name   : 开启极致RTO，业务压力中时，备机异常退出，业务压力后，备机进行重建时，注入主机CPU不足故障
Create At   : 2020/10/22
Owner       : @peilinqian
Description :
    1、开启极致RTO
    2、运行TPCC
    3、tpcc运行过程中，备机数据库进程kill - 9；查询备机数据数据进程状态
    4、查询主备状态
    5、备机进行重建
    6、注入主机CPU不足故障
    7、执行完成后，关闭极致RTO
    8、查询主备数据是否一致
Expect      :
    1、开启极致RTO成功
    2、运行TPCC正常
    3、备机数据库进程kill - 9；查询备机数据数据进程状态为空
    4、查询主备状态备机stop
    5、备机开始进行重建
    6、注入主机CPU不足故障成功，重建成功
    7、执行完成后，关闭极致RTO成功
    8、查询主备数据一致
History     :
    modified：2022-1-11 by 5328113;CPU注入结果为100%时，且未考虑sys使用CPU，优化CPU注入成功断言；
    modified：2021-9-13 by 5328113;优化CPU不足故障超时处理及断言；
    modified：2021-3-9 by 5328113;主备同步方法调用修改，增加超时等待时间；
    modified：2021-2-5 by 5328113;guc参数还原时指定还原值
"""

import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class ExtremeRto(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(
            '----Opengauss_Reliability_RTO_Hardware_Cpu_Case005:初始化----')

        self.pri_root = Node(node='PrimaryRoot')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.tpcc_path = macro.TPCC_PATH
        self.node_num = self.com.get_node_num(self.pri_root)
        self.log.info(self.node_num)
        self.max_time = 1200
        self.min_time = 300

    def test_main(self):
        try:
            self.log.info('----1、开启极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'replication_type=1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=off')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----2、运行TPCC----')
            tpcc_cmd = f'. runBenchmark.sh {macro.TPCC_PROPS_FILENAME}'
            tpcc_thread = ComThread(self.com.startTPCC,
                                    args=(self.pri_root,
                                          self.tpcc_path,
                                          tpcc_cmd,))
            tpcc_thread.setDaemon(True)
            tpcc_thread.start()
            time.sleep(10)

            self.log.info('----3、备机数据库进程kill - 9----')
            self.com.kill_pid_keyword('gaussdb', 9, self.sta1_dbuser)
            time.sleep(1)

            db_process = 'ps ux | grep \'M standby\' | grep -v grep'
            self.log.info(db_process)
            db_process_result = self.sta1_dbuser.sh(db_process).result()
            self.log.info(db_process_result)

            # 等待TPCC执行完成
            tpcc_thread.join(int(macro.TPCC_RUN_MAX_TIME) * 2)
            tpcc_result = tpcc_thread.get_result()
            self.log.info(tpcc_result)
            self.assertIn(self.constant.TPCC_SUCCESS_MSG, tpcc_result)

            self.log.info('----4、查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')

            self.log.info('----5、备机进行重建----')
            build_success_msg = self.constant.REBUILD_SUCCESS_MSG
            build_thread = ComThread(self.sta_1_sh.executeGsctl,
                                     args=('build',
                                           build_success_msg,))
            build_thread.setDaemon(True)
            build_thread.start()
            time.sleep(10)

            self.log.info('----5、利用cfe工具构造cpu压力100%----')
            self.log.info("----获取cpu个数----")
            cpu_num_cmd = "cat /proc/cpuinfo |grep 'processor'|wc -l"
            self.log.info(cpu_num_cmd)
            cpu_num = self.pri_root.sh(cpu_num_cmd).result()
            self.log.info(cpu_num)

            self.log.info("----利用cfe工具构造cpu压力100%----")
            # 因执行该指令后不会退出，所以使用启线程方式，在主线程结束后退出子线程
            inject_cmd = f"rCPU_Overloadal (cpuid1, cpuid2, usage) " \
                f"values (0, {int(cpu_num) - 1}, 100)"
            inject_thread = ComThread(self.com.cfe_inject,
                                      args=(self.pri_root,
                                            inject_cmd,))
            inject_thread.setDaemon(True)
            inject_thread.start()

            self.log.info("-----获取cfe注入故障结果----")
            inject_thread.join(10)
            inject_result = inject_thread.get_result()
            self.log.info(inject_result)
            time.sleep(2)
            self.log.info("-----获取cpu故障线程结果----")
            check_cmd = 'ps -ef |grep BusyCpu |grep -v grep'
            self.log.info(check_cmd)
            check_result = self.pri_root.sh(check_cmd).result()
            self.log.info(check_result)
            check_cmd = "top -b -n 1 |awk 'NR==3'"
            self.log.info(check_cmd)
            check_result = self.pri_root.sh(check_cmd).result()
            self.log.info(check_result)
            us = float(check_result.split(':')[1].split(',')[0].split()[0])
            sy = float(check_result.split(':')[1].split(',')[1].split()[0])
            self.assertTrue(us+sy > 90)

            # 等待build重建完成
            build_thread.join()
            build_result = build_thread.get_result()
            self.assertTrue(build_result)

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

        finally:
            self.log.info('----6、执行完成后，关闭极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=on')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            # 校验备机是否完成数据同步
            self.log.info('校验备机是否完成数据同步')
            flag_1 = self.sta_1_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_1)
            flag_2 = self.sta_2_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_2)

            self.log.info('----7、查询主备数据是否一致----')
            nodes_tuple = (self.pri_dbuser,
                           self.sta1_dbuser,
                           self.sta2_dbuser)
            flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
            self.assertTrue(flag)

            table_dict = self.com.format_sql_result(
                self.pri_sh.executDbSql(r'\d'))
            table_name = table_dict.get('Name')
            for name in table_name:
                select_sql = f'select count(*) from {name};'
                flag = self.com.check_data_sample_by_all(select_sql,
                                                         *nodes_tuple)
                self.assertTrue(flag)

    def tearDown(self):
        self.log.info("----this is tearDown----")
        self.log.info("----消除cpu压力故障----")
        clean_command = f"rCPU_Overloadal"
        self.log.info(clean_command)
        clean_result = self.com.cfe_clean(self.pri_root, clean_command)
        self.log.info(clean_result)
        time.sleep(2)
        check_cmd = "top -b -n 1 |awk 'NR==3'"
        self.log.info(check_cmd)
        check_result = self.pri_root.sh(check_cmd).result()
        self.log.info(check_result)

        self.log.info('----校验刷脏页是否完成----')
        self.pri_sh.check_whether_remain_dirty_page()

        self.log.info(
            '----Opengauss_Reliability_RTO_Hardware_Cpu_Case005:执行完毕----')
